'''
Created on 2011-10-26

@author: Administrator
'''
import os
import urllib2
import sqlite3
import datetime
from setting import *
class StockList():
    '''
    get A,HK,US stock list from all vendor
    '''
    def getstocklist(self,reg):
        if reg=='CH':return self.getCNstocklist()
        elif reg=='HK':return self.getHKstocklist()
        elif reg=='US':return self.getUSstocklist()
        else:print 'error reg'
    def getusstocklistfromimeigu(self):
        def getonepage(page):
            url='http://hq.imeigu.com/list.jsp?od=0&ac=0&&pagex='+str(page)
            content=urllib2.urlopen(url).read().decode('utf8')
            s=content.find('<table cellspacing="0">')
            p=content.find('</table',s)
            nlist=content[s:p].split('<tr')
            stocklist={}
            for n in nlist:
    #            print n
                tlist=n.split('<td')
                onestock=[]
                tstock=[]
                for t in tlist:
                    p=t.find('</td')
                    k=t.rfind('</a',0,p)
                    if k>0:p=k
                    s=t.rfind('>',0,p)
                    v=t[s:p].strip(' >\t\n')
                    tstock.append(v)
    #                print '[',v,']',len(v)
                    if len(tstock)>4:
                        onestock.append(tstock[1])
                        onestock.append(tstock[2])
                
            stocklist[onestock[1]]=onestock[2]
            return len(nlist)
        page=1
        ret=getonepage(page)
        print ret
    #    while ret==30:
    #        page=1+page
    #        ret=getonepage(page)         
        return []
    def getHKstocklistfrombloomberg(self):
        def parseonestock(src):
            nlist=src.split('<td')
            ret=[]
            for n in nlist:
                
                p=n.find('</td')
                k=n.rfind('</span',0,p)
                if k>0:p=k
                s=n.rfind('>',0,p)
                v=n[s:p].strip('>\n\t ')
                if len(v)>0: ret.append(v)
            return ret
            pass
        def getonepage(page=1):
            url='http://www.bloomberg.com/markets/companies/country/usa/'+str(page)+'/'
    #        url='http://www.bloomberg.com/markets/companies/country/hong-kong/'+str(page)+'/'
    #        url='http://www.bloomberg.com/markets/companies/country/china/'+str(page)+'/'
            content=urllib2.urlopen(url).read()
            s=content.find('<table class="ticker_data">')
            p=content.find('</table',s)
            nlist=content[s:p].split('<tr')
            stocklist=[]
            for n in nlist:
                ret=parseonestock(n)
                if len(ret)>0:stocklist.append(ret)
            return stocklist
        stocklist=[]
        tlist=[]
        page=1
        tlist=getonepage(page)
        print "page:",page," tlist len:",len(tlist)
        if len(tlist)==0:return 0
        else:
            for t in tlist:
                stocklist.append(t)
            while len(tlist)>0:
                page=page+1
                tlist=getonepage(page)
                print "page:",page," tlist len:",len(tlist)
                for t in tlist:
                    stocklist.append(t)           
        return stocklist
    def getAstocklistfrombloomberg(self):
        def parseonestock(src):
            nlist=src.split('<td')
            ret=[]
            for n in nlist:
                
                p=n.find('</td')
                k=n.rfind('</span',0,p)
                if k>0:p=k
                s=n.rfind('>',0,p)
                v=n[s:p].strip('>\n\t ')
                if len(v)>0: ret.append(v)
            return ret
            pass
        def getonepage(page=1):
            url='http://www.bloomberg.com/markets/companies/country/china/'+str(page)+'/'
            content=urllib2.urlopen(url).read()
            s=content.find('<table class="ticker_data">')
            p=content.find('</table',s)
            nlist=content[s:p].split('<tr')
            stocklist=[]
            for n in nlist:
                ret=parseonestock(n)
                if len(ret)>0:stocklist.append(ret)
            return stocklist
        stocklist=[]
        tlist=[]
        page=1
        tlist=getonepage(page)
        print "page:",page," tlist len:",len(tlist)
        if len(tlist)==0:return 0
        else:
            for t in tlist:
                stocklist.append(t)
         
        return stocklist
    def getHKstocklistfromSINA(self):
        url='http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHKStockData?page=1&num=8000&sort=symbol&asc=1&node=qbgg_hk&_s_r_a=page'
        nlist=urllib2.urlopen(url).read().decode('gb2312').strip('[]{}').split('},{')  
        stocklist={}  
        for n in nlist:
            tlist=n.split(',')
            rlist={}
            for t in tlist:
                plist=t.split(':')
                rlist[plist[0]]=plist[1].strip('"')
            stocklist[rlist['symbol']]=rlist

        return stocklist
    def getUSstockfromSINA(self):
        nodelist=['china_us','yysp_us','meida_us','auto_us','sales_us','finance_us','tech_us']
        urlpre='http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getUSList?page=1&num=8000&sort=chg&asc=0&_s_r_a=init&node='
        stocklist={}
        for node in nodelist:
            url=urlpre+node
            nlist=urllib2.urlopen(url).read().strip('{}[]').split('},{')
#            print node,len(nlist)
            for n in nlist:
                tlist=n.split(':')
                code=tlist[1].strip('"')
                stocklist[code]=code
#        print len(stocklist),stocklist
        return stocklist
    def getCHstocklistfromSINA(self):
        url='http://vip.stock.finance.sina.com.cn/quotes_service/api/json_v2.php/Market_Center.getHQNodeData?page=1&num=20000&sort=symbol&asc=1&node=hs_a&_s_r_a=init'
        content=urllib2.urlopen(url).read().decode('gb2312')
#        print content
        clist=content.strip(' []{}').split('},{')
#        print len(clist)
        stocklist={}
        for c in clist:
            flist=c.split(',')
            fsone={}
            for f in flist:
                tlist=f.split(':')
                fsone[tlist[0]]=tlist[1].strip('"')
            stocklist[fsone['code']]=fsone['name']
        return stocklist
    def getHKstocklist(self):
        return self.getHKstocklistfromSINA()
    def getCNstocklist(self):
        return self.getCHstocklistfromSINA()
        pass
    def getUSstocklist(self):
        return self.getUSstockfromSINA()

class StockListDB():
    def __init__(self):       
        
        dbfilename=os.path.join(stockdata_rootpath,str(datetime.date.today())+'_jobs.db3')
        print dbfilename
        if not os.path.exists(dbfilename):
            self.db=sqlite3.connect(dbfilename)
            self.c=self.db.cursor()
            self.createtablestocklist()
            self.updatestocklist()
        else:
            self.db=sqlite3.connect(dbfilename)
            self.c=self.db.cursor()
#            self.updatestocklist()
            
    def getjobslist(self):
        sql='select code,reg,filldate from stocklist order by code'
        self.c.execute(sql)
        ret=self.c.fetchall()
        jobslist={}
        for r in ret:
            jobslist[r[0]+':'+r[1]]=r[2]
        return jobslist
        pass
    def createtablestocklist(self):    
        sql='create table if not exists stocklist(code text,reg text,filldate text,primary key(code))'
        self.c=self.db.cursor()
        self.c.execute(sql)
        self.db.commit()        
    def updatestocklist(self):
        sql='select code from stocklist order by code'
        self.c.execute(sql)
        ret=self.c.fetchall()
        slist=[]  
        for r in ret:
            slist.append(r[0]) 
#        print len(slist),slist
        
        scls=StockList()
        reglist=['CH','HK','US']
        
        for reg in reglist:
            stocklist=scls.getstocklist(reg)  
                      
            sql='insert into stocklist (code,reg) values(?,?)'
            print reg,len(stocklist)
            for stock in stocklist:
                if stock in slist:pass
                else:self.c.execute(sql,(stock,reg))
            self.db.commit()
    def batchflagupdate(self,codelist):
        for code in codelist:
            sql='update stocklist set filldate=? where code=?'
            self.c.execute(sql,(str(datetime.date.today()),code))
        self.db.commit()            
    def flagonestock(self,code):
        sql='update stocklist set filldate=? where code=?'
        self.c.execute(sql,(str(datetime.date.today()),code))
        self.db.commit()
#    def dojob(self,job):
#        tlist=job.split(':')
#        code=tlist[0]
#        reg=tlist[1]
#        sdb=StockDataDB(code,reg)
#        sdb.updatestockdata()
#        self.flagonestock(code)
if __name__ == '__main__':
    pass